# coding: utf-8

import bs4
from scrapy.contrib.spiders.crawl import CrawlSpider
from ..items import MyroomItem
import scrapy


class MyroomSpider(CrawlSpider):
    #
    # 我房网 规则
    #

    name = "myroomspider"
    allowed_domains = ["wofang.com"]
    start_urls = ["http://www.wofang.com/"]

    def parse(self, response):
        data = response.body
        soup = bs4.BeautifulSoup(data, 'lxml')

        #
        # 获取 所有城市
        #

        citybox = soup.find_all('div', class_='all-con')[4]
        for child in citybox.findAll('a'):
            city = child.string
            city_url = str(child.get('href'))
            citys = {
                'website': '我房网', 'web_url': 'wofang.com',
                'city': city, 'city_id': city_url.replace('http://sale.wofang.com/', '').replace('/', '')

            }
            new_url = city_url.replace('http://sale.wofang.com/', 'http://www.wofang.com/building/')
            yield scrapy.Request(new_url, callback=self.parse_city_area, meta=citys)

    def parse_city_area(self, response):
        meta = response.meta
        data = response.body
        soup = bs4.BeautifulSoup(data, 'lxml')

        #
        # 获取 城市下面区域
        #

        areabox = soup.select('.seltion-m > dl')[0]
        areast = areabox.find_all('dd')[1]
        arealist = areast.find_all('a')
        for child in arealist:
            area = child.get_text()
            area_url = child.get('href')
            meta['area'] = area
            if area == '其他区域':
                area = '其他'
            if area != '全部':
                yield scrapy.Request(area_url, callback=self.parse_city_estate, meta=meta)

    def parse_city_estate(self, response):
        meta = response.meta
        data = response.body
        soup = bs4.BeautifulSoup(data, 'lxml')

        #
        # 获取 楼盘信息
        #

        estates = soup.find_all('div', class_='pic')
        for child in estates:
            col = child.find_all('a')
            for row in col:
                item = MyroomItem()
                estate = row.get('title')
                estate_url = row.get('href')
                estate_id = estate_url.replace('http://www.wofang.com/building/', '').replace('/', '')
                meta['estate_url'] = estate_url
                item['website'] = meta['website']
                item['web_url'] = meta['web_url']
                item['city'] = meta['city']
                item['city_id'] = meta['city_id']
                item['estate'] = estate
                item['estate_id'] = estate_id
                item['estate_url'] = estate_url
                item['area'] = meta['area']
                yield item

        #
        # 进行翻页
        #

        pages = soup.select('.page > a')
        if pages:
            page = pages[-1]
            next = page.get_text().strip()
            next_url = 'http://www.wofang.com' + page.get('href')
            if next == '下一页':
                yield scrapy.Request(next_url, callback=self.parse_city_estate, meta=meta)
